#-- coding: utf-8 --

#@File : quick_news_futunn_com.py
#@Software : PyCharm
#@Author : Silva
#@Email : for_billy@163.com
#@Time : 2023/9/30 上午9:54


import re
import math
import scrapy
import json
import uuid

from urllib.parse import urlsplit
from parsel import  Selector

from ..utils import date, over_page, date2time, prettify_css_html
from ..items import PostNewsItem, newsItemLoader, InfoNewsItem
from ..package.rules.utils import urljoin
from ..settings import PAGE_LIMIT
from ..package.rules import TitleRules, PublishDateRules, AuthorExtractor
from ..package.oss_util import OssClient

oss_client = OssClient(bizType='info')

class HankyungSpider(scrapy.Spider):
    name = 'hankyung.com'
    allowed_domains = ['hankyung.com']
    site_name = '韩国经济日报'
    title_rules = TitleRules()
    publish_date_rules = PublishDateRules()
    author_rules = AuthorExtractor()
    language = 'ko'

    # 按模块抓取
    start_urls = ['https://www.hankyung.com/globalmarket/news-globalmarket']

    def __init__(self, task_id='', *args, **kwargs):
        super().__init__(*args, **kwargs)  # <- important
        self.task_id = task_id
        self.site_type = 'info'
        self.need_proxy = False

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url, callback=self.get_page_list)


    def get_page_list(self, response):
        news_a_list = response.xpath('//div[@class="news-item"]//h2/a')
        for page_a in news_a_list:
            title = page_a.xpath('./text()').get()
            meta = {'title': title}
            yield response.follow(page_a, callback=self.parse_detail, meta=meta)

        # 翻页
        if 'page=' not in response.url:
            for num in range(2, 1500):
                next_page = f'https://www.hankyung.com/globalmarket/news-globalmarket?page={num}'
                yield from over_page(next_page, response, page_num=num, callback=self.get_page_list)

    def parse_detail(self, response):
        item = newsItemLoader(item=InfoNewsItem())
        article = response.xpath('//div[@class="article"]//div[@class="article-body"]')
        content = article.extract_first()
        title = response.meta.get('title')
        images = []
        content = prettify_css_html(content)
        item.add_value('content', content)  # 正文内容/text_content
        item.add_value('images', images)  # 正文内容/text_content
        item.add_value('title', title)  # 正文内容/text_content
        item.add_value('originalTitle', title)
        item.add_value('source', '한경글로벌마켓')  # 资讯源
        item.add_value('originalUrl', response.url)  # 原文url
        yield item.load_item()

